package to_hbase


import org.apache.spark.SparkConf
import org.apache.spark.sql.{DataFrame, SparkSession}

object EachWebcodeDistinctSourceCodeConceptIdDetailIdDetailName {
  def main(args: Array[String]): Unit = {
    val conf=new SparkConf().setAppName("EachWebcodeDistinctSourceCodeConceptIdDetailIdDetailName").setMaster("local[4]")
    val spark=SparkSession.builder().config(conf).getOrCreate()
    val frame=spark.read.json("/Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/Detaildic_IBRD.json/part-00000")
    frame.createTempView("table")
    spark.sql("select * from table").show()
    val result: DataFrame = spark.sql(
      """
        |select
        |distinct WebCode,SourceCode,ConceptId,DetailId,DetailName
        |from
        |table
        |""".stripMargin)
    result.show()
    result.coalesce(1).write.option("delimiter", "\t").csv("/Users/kelisiduofu/Desktop/IBRD")

    spark.stop()
  }
}
